﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.SharePoint;
using Microsoft.SharePoint.Taxonomy;
using Microsoft.SharePoint.Administration;
using System.IO;
using Microsoft.Office.Server.Utilities;
using System.Collections;
using System.Data;
using System.Reflection;

namespace FixSiteTerms
{
    class Program
    {
        #region Private Members
        //private static int RepeatLimit = 1;
        private static int RestartLimit = 3;
        private static string ToolVersion = GetToolVersion();
        private static string Welcome = "********************************************\n" +
                                        "*    Welcome to SP Tax Cleaner tool v{0}   *\n" +
                                        "*     Programmed by Ahmed Farag - 2012     *\n" +
                                        "********************************************\n";
        //Counters
        private static int InvalidFields = 0;
        private static int MissingTerms = 0;
        private static int RecycledTerms = 0;
        private static int Errors = 0;
        private static int Warnings = 0;
        #endregion

        static bool AreArgsValid(string[] args)
        {
            bool Valid = false;
            string[] ValidOptionalArgs = new string[] { "-RootSiteOnly", "-test", "-synconly",
                                                        "-CleanDuplicatesOnly", "-NoRestartIfContentChanged",
                                                        "-TurnOffChangesDetection", "-NoTermIdMatching", "-RestartLimit:", "-ConfirmationOff" };
            if (args.Length >= 1)
            {
                if (args[0].ToLower().Contains("http"))//Make sure first arg is a URL
                {
                    //Validate all other args
                    for (int i = 1; i < args.Length; i++)
                    {
                        if (!ArgExist(args, args[i]))
                            return false;
                    }
                    return true;
                }
            }
            return Valid;
        }
        static bool ArgExist(string[] ValidArgs, string argToCheck)
        {
            foreach (string arg in ValidArgs)
            {
                if (argToCheck.StartsWith(arg, StringComparison.OrdinalIgnoreCase))
                    return true;
            }
            return false;
        }
        static string GetCommandsValues(string[] args)
        {
            StringBuilder Commands = new StringBuilder();
            Commands.AppendFormat("\nTarget Site: {0}", args[0]);

            bool RootSiteOnly = DoesArgumentExist("-RootSiteOnly", args);
            bool Test = DoesArgumentExist("-test", args);
            bool SyncOnly = DoesArgumentExist("-synconly", args);
            bool CleanDupsOnly = DoesArgumentExist("-CleanDuplicatesOnly", args);
            //bool RepeatTillNoChange = DoesArgumentExist("-RepeatTillNoChange", args);
            //bool NoRestartIfContentChanged = DoesArgumentExist("-NoRestartIfContentChanged", args);
            bool TurnOffChangesDetection = DoesArgumentExist("-TurnOffChangesDetection", args);
            bool NoTermIdMatching = DoesArgumentExist("-NoTermIdMatching", args);
            bool ConfirmationOff = DoesArgumentExist("-ConfirmationOff", args);
            int ResLimit = 0;
            Commands.AppendFormat("\nRootSiteOnly: {0}", RootSiteOnly);
            Commands.AppendFormat("\nTest: {0}", Test);
            Commands.AppendFormat("\nSyncOnly: {0}", SyncOnly);
            Commands.AppendFormat("\nCleanDuplicatesOnly: {0}", CleanDupsOnly);
            //Commands.AppendFormat("\nNoRestartIfContentChanged: {0}", NoRestartIfContentChanged);
            if (!TurnOffChangesDetection)
                ResLimit = readArgumentValue("-RestartLimit:", args, RestartLimit);
            Commands.AppendFormat("\nRestartLimit: {0}", ResLimit);
            Commands.AppendFormat("\nTurnOffChangesDetection: {0}", TurnOffChangesDetection);
            Commands.AppendFormat("\nNoTermIdMatching: {0}", NoTermIdMatching);
            Commands.AppendFormat("\nConfirmationOff: {0}", ConfirmationOff);

            return Commands.ToString();
        }
        static void Main(string[] args)
        {
            //Show the welcome message
            string ArgsHelp = "Supply the Site Collection URL" +
                                  " parameters as follows:\n" +
                                  "\nSPTaxCleaner SITE_COLLECTION_URL [-Test] [-RootSiteOnly]" +
                                  " [-Synconly] [-CleanDuplicatesOnly]" +
                                  "[-RestartLimit:RESTART_LIMIT] " +
                                  "[-TurnOffChangesDetection] [-ConfirmationOff]";
            Console.WriteLine(Welcome, ToolVersion);
             if (DoesArgumentExist("?", args) || 
                DoesArgumentExist("/?", args) || 
                DoesArgumentExist("help", args) ||
                DoesArgumentExist("-help", args) ||
                DoesArgumentExist("/help", args))
            {
                Console.WriteLine(ArgsHelp);
                return;
            }
            if (!AreArgsValid(args))
            {
                Console.WriteLine("Invalid Arguments." + ArgsHelp);
                return;
            }
            DateTime StartTime = DateTime.Now;
            bool RootSiteOnly = DoesArgumentExist("-RootSiteOnly", args);
            bool Test = DoesArgumentExist("-test", args);
            bool SyncOnly = DoesArgumentExist("-synconly", args);
            bool CleanDupsOnly = DoesArgumentExist("-CleanDuplicatesOnly", args);
            //bool RepeatTillNoChange = DoesArgumentExist("-RepeatTillNoChange", args);
            //bool NoRestartIfContentChanged = DoesArgumentExist("-NoRestartIfContentChanged", args);
            bool TurnOffChangesDetection = DoesArgumentExist("-TurnOffChangesDetection", args);
            bool NoTermIdMatching = DoesArgumentExist("-NoTermIdMatching", args);
            bool ConfirmationOff = DoesArgumentExist("-ConfirmationOff", args);



            //if (RepeatTillNoChange)
            //    RepeatLimit = readArgumentValue("-RepeatTrialLimit:", args, RepeatLimit);
            if (TurnOffChangesDetection)
                RestartLimit = readArgumentValue("-RestartLimit:", args, RestartLimit);
            if (!ConfirmationOff)
            {
                Console.Write("SPTaxCleaner will execute your command as follows:\n" +
                                  GetCommandsValues(args) +
                                  "\n\nExecuting there commnads might update your data." +
                                  " Make sure you take backups of your content databases and Managed Metadata Dbs." +
                                  "\n\nAre you sure you want to continue (Y/N)? ");
                string result = Console.ReadLine().ToLower().Trim();
                if (result != "yes" && result != "y")
                {
                    Console.WriteLine("\n\nAlright! Lets do nothing now. Come back later when you are ready.");
                    return;
                }
            }

            try
            {
                if (SyncOnly)
                {
                    ExecuteSyncCommands(args[0]);
                }
                else
                {
                    FixSiteTerms(args[0], Test, RootSiteOnly, CleanDupsOnly, 
                    TurnOffChangesDetection, NoTermIdMatching, RestartLimit);
                }
            }
            catch (Exception Ex)
            {
                Console.ForegroundColor = ConsoleColor.Red;
                Console.WriteLine("Error executing cleanup commands. \nError Details:\n {0}\nStack Trace:\n {1}", Ex.Message, Ex.StackTrace);
                Console.ResetColor();
            }
            TimeSpan Duration = DateTime.Now.Subtract(StartTime);
            Console.WriteLine("\n\nFinished in {0} Hour(s), {1} Min(s), {2} Sec(s) at: {3}\n", Duration.Hours, Duration.Minutes, Duration.Seconds, DateTime.Now.ToLongTimeString());
            Console.WriteLine("Results:\n" +
                              "========\n\n" +
                              "Invalid Fields: {0}\n" +
                              "Missing Terms: {1}\n" +
                              "Recycled Terms: {2}\n" +
                              "Errors: {3}\n" +
                              "Warnings: {4}\n",
                              InvalidFields, MissingTerms, RecycledTerms, Errors, Warnings);

            Console.WriteLine("\nThe MMS cleanup process has finished");
        }
        #region Arguments Commands
        static bool DoesArgumentExist(string arg, string[] args)
        {
            return GetArgumentIndex(arg, args) >= 0;
        }
        static int GetArgumentIndex(string arg, string[] args)
        {
            for (int i = 0; i < args.Length; i++)
            {
                if (string.Compare(args[i], arg, true) == 0) return i;
            }
            return -1;
        }
        static int GetArgumentIndexThatsStartsWith(string arg, string[] args)
        {
            for (int i = 0; i < args.Length; i++)
            {
                if (args[i].StartsWith(arg, StringComparison.OrdinalIgnoreCase)) return i;
            }
            return -1;
        }
        static int readArgumentValue(string arg, string[] args, int defaultValue)
        {
            int value = defaultValue;
            int argIndex = -1;
            if ((argIndex = GetArgumentIndexThatsStartsWith(arg, args)) >= 0)
            {
                string[] values = args[argIndex].Split(':');
                if (values == null || values.Length != 2)
                    return value;
                int parsedValue = -1;
                if (!int.TryParse(values[1], out parsedValue))
                    return value;
                return parsedValue;
            }
            return value;
        }
        #endregion
        #region Program Logic Control
        static void ExecuteSyncCommands(string webAddress)
        {
            using (SPSite site = new SPSite(webAddress))
            {
                using (SPWeb web = site.OpenWeb())
                {
                    //Run Taxonomy Update Scheduler Job first
                    RunTaxonomyTimerJob(web.Url);
                    //Sync Hidden List
                    SyncHiddenList(web.Site);
                }
            }
        }
        static void ExecuteSyncCommands(SPWeb web)
        {
            //Run Taxonomy Update Scheduler Job first
            RunTaxonomyTimerJob(web.Url);
            //Sync Hidden List
            SyncHiddenList(web.Site);
        }
        static void FixSiteTerms(string webAddress, bool test, bool RootSiteOnly, bool CleanDupOnly
                                , bool TurnOffChangesDetection, bool NoTermIdMatching, int RestartLimit)
        {
            int RestartCount = 0;
            DateTime StartTime = DateTime.Now;
            while (true)
            {
                int CurrentTHLCount = 0;
                //int RunThreashold = 1;
                //int RunCount = 1;
                //int LastTHLCount = -1;
                bool Restart = false;
                bool NoCleanUpRequired = false;
                //Load the fields data first
                Console.WriteLine("Scanning site collection for taxonomy usage\n\nPlease wait, this will take few minutes to finish ...\n");
                long ChangesCount = 0;
                //We can't run with RootSiteOnly=true here cause for scainning the used terms section in the duplicate Cleanup
                // below we  have to get that info from everywhere even the sub sites so we don't delete a term that is being used
                //may in a sub site
                Hashtable WebsInfo = GetSitesInfo(webAddress, true, false, ref ChangesCount);
                Console.WriteLine("\n\nFound {0} site(s) and {1} term entries.\n\n", WebsInfo.Count, ChangesCount);


                Console.WriteLine("\nStarting cleanup process please wait... ");
                //Cleanup list items
                if (!CleanDupOnly && !NoCleanUpRequired)
                {
                    if (test)
                        ExecuteCleanUpTest(webAddress, WebsInfo, RootSiteOnly, true);
                    else
                        ExecuteCleanUp(webAddress, WebsInfo, RootSiteOnly, true, NoTermIdMatching);
                }
                NoCleanUpRequired = true;
                ExecuteSyncCommands(webAddress);
                //Clear Duplicates
                Console.WriteLine("\nGetting taxonomy usage info ...\n");

                StartTime = DateTime.Now;
                HashSet<int> UsedTermIds = GetUsedTermsTHLids(webAddress, WebsInfo, false);
                if (UsedTermIds == null || UsedTermIds.Count <= 0)
                {
                    WriteError("\nCannot find any used terms. Duplicates removal is canceled to protect the Taxnonmy Hidden List\n");
                    break;
                }
                else
                {
                    //Check if site collection has changed
                    bool Changed = GetSiteLastChangDate(webAddress) > StartTime;
                    //Check if the site data has changed since the process has started
                    if (!TurnOffChangesDetection && Changed)
                    {
                        WriteWarning("The site collection data has been modified. The duplicates removal process will be terminated.");
                        Restart = true;
                        WriteWarning("The cleanup process will be restarted to capture the new changes.\n\n");
                        break;
                    }
                    Console.WriteLine("\nRecycling unused terms from the Taxonomy Hidden List ...\n");
                    if(test)
                        CurrentTHLCount = DeleteDuplicatesTest(UsedTermIds, webAddress);
                            else
                        CurrentTHLCount = DeleteDuplicates(UsedTermIds, webAddress);
                    
                }

                if (!Restart)
                {
                    break;
                }
                else
                {
                    if (RestartCount >= RestartLimit)
                    {
                        WriteWarning("The restart limit has been reached the process will terminate.");
                        break;
                    }
                    RestartCount++;
                }
            }

        }
        #endregion
        #region Cleanup commands
        static void ExecuteCleanUp(string webAddress, Hashtable webInfo, bool RootSiteOnly, bool RunSyncCommands, bool NoTermIdMatching)
        {
            using (SPSite site = new SPSite(webAddress))
            {
                //Cleanup all webs
                for (int i = 0; i < site.AllWebs.Count; i++)
                {
                    using (SPWeb web = site.AllWebs[i])
                    {
                        ExecuteCleanUp(web, webInfo, RootSiteOnly, RunSyncCommands, NoTermIdMatching);
                    }
                    if (RootSiteOnly) return;
                }
            }
        }
        static bool ExecuteCleanUp(SPWeb web, Hashtable webInfo, bool RootSiteOnly, bool RunSyncCommands, bool NoTermIdMatching)
        {
            //Run Taxonomy Update Scheduler Job first
            if (RunSyncCommands)
                RunTaxonomyTimerJob(web.Url);
            //Sync Hidden List
            if (web != null && RunSyncCommands)
                SyncHiddenList(web.Site);

            TaxonomySession taxonomySession = new TaxonomySession(web.Site);
            //Type taxonomyType = typeof(TaxonomyField);

            Console.ForegroundColor = ConsoleColor.Magenta;
            Console.WriteLine("Cleaning web: {0}", web.Url);
            Console.ResetColor();
            Dictionary<Guid, List<FieldInfo>> ListsInfo = webInfo[web.ID.ToString()] as Dictionary<Guid, List<FieldInfo>>;
            if (ListsInfo == null)
            {
                Errors++;
                WriteError("Unable to retrieve the list info table for the web {0}", web.Url);
                return false;
            }

            foreach (KeyValuePair<Guid, List<FieldInfo>> ListInfo in ListsInfo)
            {

                //Get the current list
                SPList list = web.Lists[ListInfo.Key];
                try
                {
                    SPQuery query = new SPQuery();
                    //Get into subfolders if any
                    query.ViewAttributes = "Scope=\"RecursiveAll\"";
                    query.QueryThrottleMode = SPQueryThrottleOption.Override;
                    string WhereCondition = BuildCamlQuery(ListInfo.Value);
                    query.Query = WhereCondition + ContentIterator.ItemEnumerationOrderByID;
                    ContentIterator CIitems = new ContentIterator();
                    HashSet<Guid> FieldsBlackList = new HashSet<Guid>();
                    CIitems.ProcessListItems(list, query, delegate(SPListItem item)
                    {
                        bool NeedsToCheckIn = false;
                        bool NeedsToBePublished = false;
                        bool ReadyForUpdate = false;
                        bool NeedsToBeApproved = false;
                        bool NeedsToBeRejected = false;
                        foreach (FieldInfo FieldInfoObj in ListInfo.Value)
                        {
                            //Skip blacklisted fields
                            if (FieldsBlackList.Contains(FieldInfoObj.ID)) continue;

                            TaxonomyField TaxField = list.Fields[FieldInfoObj.ID] as TaxonomyField;

                            //Igonre invaild fields
                            if (!FieldInfoObj.Valid)
                            {

                                WriteWarning("Field: '{0}' in the list: '{2}' @ '{1}' references a TermSet with the ID='{3}' that doesn't exist in the term store or has a different ID." +
                                " Try to fix this field reference for better clean up results and to prevent future deuplicates in the taxonomy hidden list.\n",
                                             FieldInfoObj.InternalName, web.Url + list.DefaultViewUrl, list.Title, TaxField.TermSetId);
                                FieldsBlackList.Add(FieldInfoObj.ID);
                                Warnings++;
                                InvalidFields++;
                                continue;
                            }

                            //Get current field Term Set ID
                            Guid TermSetID = TaxField.TermSetId;


                            object tempFieldData = item[FieldInfoObj.ID];

                            if (tempFieldData != null && !string.IsNullOrEmpty(tempFieldData.ToString()))
                            {
                                TaxonomyFieldValueCollection currentTerms = null;
                                if (TaxField.AllowMultipleValues)
                                {
                                    currentTerms = tempFieldData as TaxonomyFieldValueCollection;
                                }
                                else
                                {
                                    TaxonomyFieldValue currentTerm = tempFieldData as TaxonomyFieldValue;
                                    currentTerms = new TaxonomyFieldValueCollection(TaxField);
                                    currentTerms.Add(currentTerm);
                                }
                                TaxonomyFieldValueCollection newTerms = new TaxonomyFieldValueCollection(TaxField);


                                //if (test)
                                //{
                                //    Console.WriteLine("Term Replacement Starting");
                                //}

                                foreach (TaxonomyFieldValue term in currentTerms)
                                {
                                    //Console.WriteLine(term);
                                    //Get the term, term set ID


                                    string fullTermLabel = term.Label;


                                    int lastIndexOfTermSeparator = fullTermLabel.LastIndexOf(':');
                                    string termLabel = fullTermLabel.Substring(lastIndexOfTermSeparator + 1);
                                    bool termIsFound = false;
                                    Term matchedTerm = null;
                                    if (NoTermIdMatching)
                                    {
                                        TermCollection existingTerms = taxonomySession.GetTerms(termLabel, false);

                                        if (existingTerms.Count >= 1) //If you found the used term in the term store lets verify it belongs to the same path and the right term set
                                        {

                                            foreach (Term foundTerm in existingTerms)
                                            {
                                                string foundTermPath = foundTerm.GetPath().Replace(";", ":");

                                                if (foundTermPath == fullTermLabel &&
                                                    foundTerm.TermSet.Id == TermSetID)
                                                {
                                                    termIsFound = true;
                                                    matchedTerm = foundTerm;
                                                    break;
                                                }
                                            }



                                        }
                                    }
                                    else
                                    {
                                        matchedTerm = taxonomySession.GetTerm(new Guid(term.TermGuid));
                                        if (matchedTerm != null)
                                            termIsFound = true;

                                    }
                                    if (termIsFound)
                                    {
                                        // Console.WriteLine("Replacing Term: {0} New Term: {1}", fullTermLabel, matchedTerm.GetPath());
                                        string termValueAsString = "-1" + ";#" + matchedTerm.GetPath() + TaxonomyField.TaxonomyGuidLabelDelimiter + matchedTerm.Id.ToString();
                                        //Console.WriteLine(termValueAsString);
                                        TaxonomyFieldValue termValue = new TaxonomyFieldValue(termValueAsString);
                                        newTerms.Add(termValue);
                                    }
                                    else
                                    {
                                        MissingTerms++;
                                        Console.WriteLine("The term '{0}' for the item: '{1}' with ID: '{2}' at the List: '{3}' @ located at '{4}' has no matching term in the store.\n",
                                            fullTermLabel, item.Title, item.ID, list.Title, web.Site.Url + list.DefaultViewUrl);
                                    }

                                }

                                try
                                {
                                    if (newTerms.Count > 0)
                                    {
                                        //Prep item for update
                                        if (!ReadyForUpdate)
                                        {
                                            //Check if the file requires approval
                                            if (item.ModerationInformation != null)
                                            {
                                                NeedsToBeApproved = item.ModerationInformation.Status == SPModerationStatusType.Approved ||
                                                                    item.ModerationInformation.Status == SPModerationStatusType.Scheduled;
                                                NeedsToBeRejected = item.ModerationInformation.Status == SPModerationStatusType.Denied;
                                            }
                                            //requires check-out and is not checked out yet
                                            if (item.File != null && item.File.RequiresCheckout && item.File.CheckOutType == SPFile.SPCheckOutType.None)
                                            {
                                                if (item.ModerationInformation != null && (
                                                   item.ModerationInformation.Status == SPModerationStatusType.Scheduled ||
                                                   item.ModerationInformation.Status == SPModerationStatusType.Approved ||
                                                   item.ModerationInformation.Status == SPModerationStatusType.Denied ||
                                                   item.ModerationInformation.Status == SPModerationStatusType.Pending))
                                                    NeedsToBePublished = true;
                                                if (item.File.Level == SPFileLevel.Published)
                                                    NeedsToBePublished = true;
                                                NeedsToCheckIn = true;
                                                item.File.CheckOut();
                                            }//requires check-out and checked-out to someone else
                                            else if (item.File != null && item.File.RequiresCheckout && item.File.CheckOutType != SPFile.SPCheckOutType.None)
                                            {
                                                if (item.File.CheckedOutByUser.LoginName.ToUpper() != web.CurrentUser.LoginName.ToUpper())
                                                {
                                                    item.File.CheckIn("Checked-In by MMS Doc Tool to prepare for cleanup", SPCheckinType.MinorCheckIn);
                                                    item.File.CheckOut();
                                                    NeedsToCheckIn = true;
                                                }
                                            }
                                            ReadyForUpdate = true;
                                        }
                                        //Update the field values
                                        if (TaxField.AllowMultipleValues && newTerms.Count > 0)
                                        {
                                            TaxField.SetFieldValue(item, newTerms);
                                        }
                                        else if (newTerms.Count > 0 && newTerms.Count > 0)
                                        {
                                            TaxField.SetFieldValue(item, newTerms[0]);
                                        }
                                    }
                                }
                                catch (Exception Ex)
                                {
                                    WriteError("Error updating item with Title: {0}, ID: {1}. Error Details: {2}, Stack Trace: {3}", item.Title, item.ID, Ex.Message, Ex.StackTrace);
                                    Errors++;
                                }
                            }

                        }
                        try
                        {

                            if (ReadyForUpdate && !NeedsToCheckIn)
                            {
                                item.Update();
                            }

                            if (NeedsToCheckIn)
                            {
                                if (NeedsToBePublished)
                                {
                                    item.File.CheckIn("Checked-In by SP Tax Cleaner Tool after cleanup", SPCheckinType.MajorCheckIn);
                                    if (NeedsToBeApproved)
                                        item.File.Approve("Approved by SP Tax Cleaner Tool after cleanup");
                                    if (NeedsToBeRejected)
                                        item.File.Deny("Rejected by SP Tax Cleaner Tool after cleanup");
                                }
                                else
                                {
                                    item.File.CheckIn("Checked-In by SP Tax Cleaner Tool after cleanup", SPCheckinType.MinorCheckIn);
                                }
                            }
                            //Console.ForegroundColor = ConsoleColor.Green;
                            //Console.WriteLine("Term Replacement Finished\n");
                            //Console.ResetColor();

                        }
                        catch (Exception Ex)
                        {
                            WriteError("Error updating (check-in\\Publish) item with Title: {0}, ID: {1}. Error Details: {2}, Stack Trace: {3}", item.Title, item.ID, Ex.Message, Ex.StackTrace);
                            Errors++;
                        }
                    }, null);
                }
                catch (Exception Ex)
                {
                    WriteError("Error while executing clean up routins. Error Details: {0}, Stack Trace: {1}", Ex.Message, Ex.StackTrace);
                    Errors++;
                }
            }
            return true;
        }
        #region Cleanup commands Testing
        static void ExecuteCleanUpTest(string webAddress, Hashtable webInfo, bool RootSiteOnly, bool RunSyncCommands)
        {
            using (SPSite site = new SPSite(webAddress))
            {
                using (SPWeb web = site.OpenWeb())
                {
                    ExecuteCleanUpTest(web, webInfo, RootSiteOnly, RunSyncCommands);
                }
            }
        }
        static bool ExecuteCleanUpTest(SPWeb web, Hashtable webInfo, bool RootSiteOnly, bool RunSyncCommands)
        {
            //Run Taxonomy Update Scheduler Job first
            if (RunSyncCommands)
                RunTaxonomyTimerJob(web.Url);
            //Sync Hidden List
            if (web != null && RunSyncCommands)
                SyncHiddenList(web.Site);

            TaxonomySession taxonomySession = new TaxonomySession(web.Site);
            //Type taxonomyType = typeof(TaxonomyField);

            Console.ForegroundColor = ConsoleColor.Magenta;
            Console.WriteLine("Cleaning web: {0}", web.Url);
            Console.ResetColor();
            Dictionary<Guid, List<FieldInfo>> ListsInfo = webInfo[web.ID.ToString()] as Dictionary<Guid, List<FieldInfo>>;
            if (ListsInfo == null)
            {
                WriteError("Unable to retrieve the list info table for the web {0}", web.Url);
                return false;
            }

            foreach (KeyValuePair<Guid, List<FieldInfo>> ListInfo in ListsInfo)
            {

                //Get the current list
                SPList list = web.Lists[ListInfo.Key];
                try
                {

                    SPQuery query = new SPQuery();
                    //Get into subfolders if any
                    query.ViewAttributes = "Scope=\"RecursiveAll\"";
                    query.QueryThrottleMode = SPQueryThrottleOption.Override;
                    query.Query = BuildCamlQuery(ListInfo.Value) + ContentIterator.ItemEnumerationOrderByID;
                    ContentIterator CIitems = new ContentIterator();
                    CIitems.ProcessListItems(list, query, delegate(SPListItem item)
                    {
                        foreach (FieldInfo FieldInfoObj in ListInfo.Value)
                        {
                            //Ignore invalid fields
                            if (!FieldInfoObj.Valid)
                            {
                                Console.WriteLine("Term set is invalid for field '{0}' in list '{1}'", FieldInfoObj.InternalName, list.Title);
                                continue;
                            }
                            TaxonomyField TaxField = list.Fields[FieldInfoObj.ID] as TaxonomyField;
                            object tempFieldData = item[FieldInfoObj.ID];

                            if (tempFieldData != null && !string.IsNullOrEmpty(tempFieldData.ToString()))
                            {
                                TaxonomyFieldValueCollection currentTerms = null;
                                if (TaxField.AllowMultipleValues)
                                {
                                    currentTerms = tempFieldData as TaxonomyFieldValueCollection;
                                }
                                else
                                {
                                    TaxonomyFieldValue currentTerm = tempFieldData as TaxonomyFieldValue;
                                    currentTerms = new TaxonomyFieldValueCollection(TaxField);
                                    currentTerms.Add(currentTerm);
                                }
                                TaxonomyFieldValueCollection newTerms = new TaxonomyFieldValueCollection(TaxField);

                                Console.WriteLine("Term Replacement Starting");


                                foreach (TaxonomyFieldValue term in currentTerms)
                                {
                                    //Console.WriteLine(term);
                                    string fullTermLabel = term.Label;


                                    int lastIndexOfTermSeparator = fullTermLabel.LastIndexOf(':');
                                    string termLabel = fullTermLabel.Substring(lastIndexOfTermSeparator + 1);

                                    TermCollection existingTerms = taxonomySession.GetTerms(termLabel, false);

                                    if (existingTerms.Count > 1)
                                    {

                                        Console.ForegroundColor = ConsoleColor.Green;
                                        Console.WriteLine("Found Terms");
                                        Console.ResetColor();


                                        bool termIsFound = false;

                                        Term matchedTerm = null;

                                        foreach (Term foundTerm in existingTerms)
                                        {
                                            string foundTermPath = foundTerm.GetPath().Replace(";", ":");

                                            if (foundTermPath == fullTermLabel)
                                            {
                                                Console.ForegroundColor = ConsoleColor.Green;
                                                Console.WriteLine("Matching Term Found Found Term: {0} Full Term: {1}", foundTermPath, fullTermLabel);
                                                Console.ResetColor();
                                                termIsFound = true;
                                                matchedTerm = foundTerm;
                                            }
                                        }

                                        if (!termIsFound)
                                        {
                                            Console.ForegroundColor = ConsoleColor.Yellow;
                                            Console.WriteLine("No exact term found");
                                            Console.ResetColor();
                                        }
                                        else if (termIsFound)
                                        {
                                            Console.WriteLine("Replacing Term: {0} New Term: {1}", fullTermLabel, matchedTerm.GetPath());
                                            string termValueAsString = "-1" + ";#" + matchedTerm.GetPath() + TaxonomyField.TaxonomyGuidLabelDelimiter + matchedTerm.Id.ToString();
                                            Console.WriteLine(termValueAsString);
                                            TaxonomyFieldValue termValue = new TaxonomyFieldValue(termValueAsString);
                                            newTerms.Add(termValue);
                                        }


                                        Console.WriteLine("End of Found Terms");

                                    }
                                    else if (existingTerms.Count <= 0)
                                    {
                                        Console.WriteLine("Term For Item '{0}' At List '{1}' Located At {2} Has No Matching Term", item.ID, list.Title, list.DefaultViewUrl);
                                    }
                                    else
                                    {
                                        try
                                        {
                                            Console.WriteLine("    Old Term: {0} New Term: {1}", term.Label, existingTerms[0].GetPath());
                                        }
                                        catch
                                        {
                                            Console.WriteLine("Error Occurred Full Term Label: {0} | Term Label: {1} | Found Terms Count: {2}", fullTermLabel, termLabel, existingTerms.Count);
                                        }
                                    }
                                }
                                /*
                                try
                                {
                                    if (newTerms.Count > 0)
                                    {
                                        //Prep item for update
                                        if (!ReadyForUpdate)
                                        {
                                            //Check if the file requires approval
                                            if (item.ModerationInformation != null)
                                            {
                                                NeedsToBeApproved = item.ModerationInformation.Status == SPModerationStatusType.Approved ||
                                                                    item.ModerationInformation.Status == SPModerationStatusType.Scheduled;
                                                NeedsToBeRejected = item.ModerationInformation.Status == SPModerationStatusType.Denied;
                                            }
                                            //requires check-out and is not checked out yet
                                            if (item.File != null && item.File.RequiresCheckout && item.File.CheckOutType == SPFile.SPCheckOutType.None)
                                            {
                                                if (item.ModerationInformation != null && (
                                                   item.ModerationInformation.Status == SPModerationStatusType.Scheduled ||
                                                   item.ModerationInformation.Status == SPModerationStatusType.Approved ||
                                                   item.ModerationInformation.Status == SPModerationStatusType.Denied ||
                                                   item.ModerationInformation.Status == SPModerationStatusType.Pending))
                                                    NeedsToBePublished = true;
                                                if (item.File.Level == SPFileLevel.Published)
                                                    NeedsToBePublished = true;
                                                NeedsToCheckIn = true;
                                                item.File.CheckOut();
                                            }//requires check-out and checked-out to someone else
                                            else if (item.File != null && item.File.RequiresCheckout && item.File.CheckOutType != SPFile.SPCheckOutType.None)
                                            {
                                                if (item.File.CheckedOutByUser.LoginName.ToUpper() != web.CurrentUser.LoginName.ToUpper())
                                                {
                                                    item.File.CheckIn("Checked-In by MMS Doc Tool to prepare for cleanup", SPCheckinType.MinorCheckIn);
                                                    item.File.CheckOut();
                                                    NeedsToCheckIn = true;
                                                }
                                            }
                                            ReadyForUpdate = true;
                                        }
                                        //Update the field values
                                        if (TaxField.AllowMultipleValues && newTerms.Count > 0)
                                        {
                                            TaxField.SetFieldValue(item, newTerms);
                                        }
                                        else if (newTerms.Count > 0 && newTerms.Count > 0)
                                        {
                                            TaxField.SetFieldValue(item, newTerms[0]);
                                        }
                                    }
                                }
                                catch (Exception Ex)
                                {
                                    Console.ForegroundColor = ConsoleColor.Red;
                                    Console.WriteLine(Ex.Message);
                                    Console.ResetColor();
                                }*/

                            }
                            else
                            {
                                //Console.ForegroundColor = ConsoleColor.Yellow;
                                //Console.WriteLine("Item Is Empty: {0} {1}", item.ID, item.Title);
                                //Console.ResetColor();
                            }
                        }

                        Console.ForegroundColor = ConsoleColor.Green;
                        Console.WriteLine("Term Replacement Test Finished Successfully");
                        Console.ResetColor();

                    }, null);
                }
                catch (Exception Ex)
                {
                    Console.WriteLine(Ex.Message);
                }
            }

            if (!RootSiteOnly)
            {
                foreach (SPWeb childWeb in web.Webs)
                {
                    ExecuteCleanUpTest(childWeb, webInfo, RootSiteOnly, false);
                    if (childWeb != null)
                        childWeb.Dispose();
                }
            }
            return true;
        }
        #endregion
        #endregion
        #region Screen Messages Control
        static void WriteError(string format, params object[] args)
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine("Error: " + format, args);
            Console.ResetColor();
        }
        static void WriteError(string format, object arg)
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine("Error: " + format, arg);
            Console.ResetColor();
        }
        static void WriteError(string Message)
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine("Error: " + Message);
            Console.ResetColor();
        }
        static void WriteWarning(string format, params object[] args)
        {
            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine(format, args);
            Console.ResetColor();
        }
        static void WriteWarning(string format, object arg)
        {
            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine(format, arg);
            Console.ResetColor();
        }
        static void WriteWarning(string Message)
        {
            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine(Message);
            Console.ResetColor();
        }
        static void WriteSuccess(string format, params object[] args)
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine(format, args);
            Console.ResetColor();
        }
        static void WriteSuccess(string format, object arg)
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine(format, arg);
            Console.ResetColor();
        }
        static void WriteSuccess(string Message)
        {
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine(Message);
            Console.ResetColor();
        }
        #endregion
        #region Delete Duplicates Commands
        static DateTime GetSiteLastChangDate(string webAddress)
        {
            using (SPSite site = new SPSite(webAddress))
            {
                return site.LastContentModifiedDate.ToLocalTime();
            }
        }
        static HashSet<int> GetUsedTermsTHLids(SPWeb web, Hashtable WebsInfo, bool RunSyncCommands)
        {

            HashSet<int> UsedItemsFromTHL = new HashSet<int>();
            //Run Taxonomy Update Scheduler Job first & Sync Hidden List
            if (RunSyncCommands)
                ExecuteSyncCommands(web);

            //TaxonomySession taxonomySession = new TaxonomySession(web.Site);
            Type taxonomyType = typeof(TaxonomyField);

            Console.ForegroundColor = ConsoleColor.Magenta;
            Console.WriteLine("Scanning web: {0}", web.Url);
            Console.ResetColor();

            Dictionary<Guid, List<FieldInfo>> ListsInfo = WebsInfo[web.ID.ToString()] as Dictionary<Guid, List<FieldInfo>>;
            if (ListsInfo == null)
            {
                WriteError("Unable to retrieve the list info table for the web {0}. Make sure that list doesn't" +
                    " contain any references to terms or it will lose all the referenced terms from the Taxonomy Hidden List", web.Url);
                Errors++;
                return UsedItemsFromTHL;
            }

            foreach (KeyValuePair<Guid, List<FieldInfo>> ListInfo in ListsInfo)
            {
                //Get the current list
                SPList list = web.Lists[ListInfo.Key];
                SPQuery query = new SPQuery();
                query.ViewAttributes = "Scope=\"RecursiveAll\"";
                query.QueryThrottleMode = SPQueryThrottleOption.Override;
                query.Query = BuildCamlQuery(ListInfo.Value) + ContentIterator.ItemEnumerationOrderByID;
                ContentIterator listCI = new ContentIterator();
                listCI.ProcessListItems(list, query, delegate(SPListItem item)
                {
                    foreach (FieldInfo FieldInfoObj in ListInfo.Value)
                    {
                        TaxonomyField TaxField = list.Fields[FieldInfoObj.ID] as TaxonomyField;
                        object tempFieldData = item[FieldInfoObj.ID];

                        if (tempFieldData != null && !string.IsNullOrEmpty(tempFieldData.ToString()))
                        {
                            if (TaxField.AllowMultipleValues)
                            {
                                TaxonomyFieldValueCollection currentTerms = tempFieldData as TaxonomyFieldValueCollection;
                                if (currentTerms == null)
                                {
                                    Console.ForegroundColor = ConsoleColor.Red;
                                    Console.WriteLine("Null currentTerms Current Term Set is Invalid: {0} Field Name: {1}", TaxField.TermSetId, TaxField.InternalName);
                                    Console.ResetColor();
                                    Errors++;
                                }
                                else
                                {
                                    foreach (TaxonomyFieldValue term in currentTerms)
                                    {
                                        UsedItemsFromTHL.Add(term.WssId);
                                    }
                                }
                            }
                            else
                            {
                                TaxonomyFieldValue currentTerm = item[FieldInfoObj.ID] as TaxonomyFieldValue;
                                if (currentTerm == null)
                                {
                                    Console.ForegroundColor = ConsoleColor.Red;
                                    Console.WriteLine("Null currentTerm Current Term Set is Invalid: {0} Field Name: {1}", TaxField.TermSetId, TaxField.InternalName);
                                    Console.ResetColor();
                                    Errors++;
                                }
                                else
                                    UsedItemsFromTHL.Add(currentTerm.WssId);
                            }
                        }
                    }


                },
                delegate(SPListItem item, Exception e)
                {
                    throw e;
                });
            }
            return UsedItemsFromTHL;
        }
        static HashSet<int> GetUsedTermsTHLids(string webAddress, Hashtable WebsInfo, bool RunSyncCommands)
        {
            HashSet<int> UsedItemsFromTHL = new HashSet<int>();
            using (SPSite site = new SPSite(webAddress))
            {
                for (int i = 0; i < site.AllWebs.Count; i++)
                {
                    using (SPWeb web = site.AllWebs[i])
                    {
                        HashSet<int> TempUsedIdslist = GetUsedTermsTHLids(web, WebsInfo, RunSyncCommands);
                        foreach (int itemId in TempUsedIdslist)
                        {
                            UsedItemsFromTHL.Add(itemId);
                        }
                    }
                }
            }
            return UsedItemsFromTHL;
        }
        private static int DeleteDuplicatesTest(HashSet<int> UsedItemIDs, string rootWebAddress)
        {
            int THLitemCount = -1;
            bool DupFound = false;
            using (SPSite site = new SPSite(rootWebAddress))
            {
                SPList THL = site.RootWeb.Lists.TryGetList("TaxonomyHiddenList");
                if (THL == null)
                    throw new FileNotFoundException("The 'Taxonomy Hidden List' is not found. Make sure that you are accessing the root web" +
                                                    "and that the parent site and its children webs has a list that uses MMS", "TaxonomyHiddenList");

                THLitemCount = THL.ItemCount;
                ContentIterator THLiterator = new ContentIterator();
                THLiterator.ProcessListItems(THL, delegate(SPListItem item)
                {
                    //If the term is not used. Recycle the term.
                    if (!UsedItemIDs.Contains(item.ID))
                    {
                        WriteSuccess("Term with Title: {0}, ID: {1} will be recycled", item.Title, item.ID);


                        DupFound = true;
                    }
                }
                , delegate(SPListItem item, Exception e)
                {
                    Console.ForegroundColor = ConsoleColor.Red;
                    Console.WriteLine("Error recycling the term with Title: {0}, ID: {1}. Error Details: {2}", item.Title, item.ID, e.Message);
                    Console.ResetColor();
                    return false;
                });
                if (!DupFound)
                {
                    //Console.ForegroundColor = ConsoleColor.Green;
                    //Console.WriteLine("No duplicates found");
                    //Console.ResetColor();
                    return -99; //Code for no more duplicates are found
                }

            }
            return THLitemCount;

        }
        private static int DeleteDuplicates(HashSet<int> UsedItemIDs, string rootWebAddress)
        {
            int THLitemCount = -1;
            bool DupFound = false;
            using (SPSite site = new SPSite(rootWebAddress))
            {
                SPList THL = site.RootWeb.Lists.TryGetList("TaxonomyHiddenList");
                if (THL == null)
                    throw new FileNotFoundException("The 'Taxonomy Hidden List' is not found. Make sure that you are accessing the root web" +
                                                    "and that the parent site and its children webs has a list that uses MMS", "TaxonomyHiddenList");
                SPQuery query = new SPQuery();
                query.QueryThrottleMode = SPQueryThrottleOption.Override;
                SPListItemCollection Items = THL.GetItems(query);
                THLitemCount = THL.ItemCount;
                int ItemID = -1;
                string ItemTitle = "";
                try
                {

                    for (int i = 0; i < Items.Count; i++)
                    {
                        SPListItem item = Items[i];


                        //If the term is not used. Recycle the term.
                        if (!UsedItemIDs.Contains(item.ID))
                        {
                            ItemID = item.ID;
                            ItemTitle = item.Title;
                            if (string.IsNullOrEmpty(item.Title))
                                WriteSuccess("Term with no title & ID: {0} has been recycled", item.ID);
                            else
                                WriteSuccess("Term with Title: {0} & ID: {1} has been recycled", item.Title, item.ID);
                            item.Recycle();
                            i--;
                            THLitemCount--;
                            RecycledTerms++;
                            DupFound = true;
                        }

                    }
                }
                catch (Exception Ex)
                {
                    Console.ForegroundColor = ConsoleColor.Red;
                    Console.WriteLine("Error recycling the term with Title: {0}, ID: {1}. Error Details: {2}. Stack Trace: {3}", ItemTitle, ItemID, Ex.Message, Ex.StackTrace);
                    Console.ResetColor();
                    Errors++;
                }
                if (!DupFound)
                {
                    //Console.ForegroundColor = ConsoleColor.Green;
                    //Console.WriteLine("No duplicates found");
                    //Console.ResetColor();
                    return -99; //Code for no more duplicates are found
                }

            }
            return THLitemCount;

        }
        #endregion
        #region Taxonomy Sync Commands
        private static void RunTaxonomyTimerJob(string webAddress)
        {
            SPWebApplication webApp = SPWebApplication.Lookup(new Uri(webAddress));
            SPJobDefinition taxonomyJob = null;
            DateTime currTime = System.DateTime.Now;

            foreach (SPJobDefinition job in webApp.JobDefinitions)
            {
                if (job is HiddenListFullSyncJobDefinition)
                {
                    currTime = System.DateTime.Now;
                    taxonomyJob = job;

                    taxonomyJob.RunNow();
                    Console.ForegroundColor = ConsoleColor.Green;
                    Console.WriteLine("Taxonomy TimerJob Exists and has been run");
                    Console.ResetColor();
                    break;
                }
            }
            Console.ForegroundColor = ConsoleColor.Yellow;
            Console.WriteLine("Waiting 1 second(s) for Taxonomy Update Scheduler timer job to finish");
            Console.ResetColor();
            System.Threading.Thread.Sleep(new TimeSpan(0, 0, 1));

            if (taxonomyJob != null)
            {
                if (taxonomyJob.LastRunTime > currTime)
                {
                    return;
                }
                else
                {
                    System.Threading.Thread.Sleep(new TimeSpan(0, 0, 1));
                    Console.ForegroundColor = ConsoleColor.Yellow;
                    Console.WriteLine("Waiting 1 more second(s) for Taxonomy Update Scheduler timer job to finish");
                    Console.ResetColor();
                    return;
                }
            }

        }
        private static void SyncHiddenList(SPSite site)
        {
            Console.WriteLine("Start Taxonomy Hidden List Sync Command ...");
            TaxonomySession.SyncHiddenList(site);
            Console.ForegroundColor = ConsoleColor.Green;
            Console.WriteLine("Taxonomy Hidden List sync command finished successfully");
            Console.ResetColor();
        }
        #endregion
        #region Site Survey Commands
        private static Hashtable GetSitesInfo(string webAddress, bool ExcludeSystemLists, bool RootSiteOnly, ref long RequiredChangesCount)
        {
            Hashtable Webs = new Hashtable();
            using (SPSite site = new SPSite(webAddress))
            {
                for (int i = 0; i < site.AllWebs.Count; i++)
                {
                    using (SPWeb web = site.AllWebs[i])
                    {
                        Hashtable ListsInfoChild = GetListsInfo(web, ExcludeSystemLists, RootSiteOnly, ref RequiredChangesCount);
                        foreach (var key in ListsInfoChild.Keys)
                        {
                            Webs.Add(key, ListsInfoChild[key]);
                        }
                        if (RootSiteOnly) break;
                    }
                }
            }
            return Webs;
        }
        private static Hashtable GetListsInfo(SPWeb web, bool ExcludeSystemLists, bool RootSiteOnly, ref long dataEntriesCount)
        {
            Console.WriteLine("Scanning site '{0}' @ '{1}'", web.Title, web.Url);
            Hashtable WebsInfo = new Hashtable();
            Dictionary<Guid, List<FieldInfo>> Lists = new Dictionary<Guid, List<FieldInfo>>();
            List<FieldInfo> Fields = null;
            Type taxonomyType = typeof(TaxonomyField);
            foreach (SPList list in web.Lists)
            {
                try
                {
                    //Exclude system lists
                    if (ExcludeSystemLists && !list.AllowDeletion && list.Hidden)
                    {
                        WriteWarning("The List '{0}' is a system list and has been skipped", list.Title);
                        continue;
                    }
                    //Lists = new Dictionary<Guid, List<FieldInfo>>();
                    Fields = new List<FieldInfo>();

                    //Search for lists with Tax Fields
                    foreach (SPField field in list.Fields)
                    {
                        if (field.GetType() == taxonomyType)
                        {
                            TaxonomyField TaxField = field as TaxonomyField;
                            //Add the found tax fields to the list
                            Fields.Add(new FieldInfo(field.Id, TaxField.IsTermSetValid, TaxField.InternalName));
                        }
                    }
                    if (Fields.Count > 0)
                    {
                        //Add the list entry with its field info
                        Lists.Add(list.ID, Fields);

                        //Calculate count
                        dataEntriesCount += Convert.ToInt64(list.ItemCount) * Convert.ToInt64(Fields.Count);
                    }
                }
                catch (Exception Ex)
                {
                    WriteWarning("Unable to load the list '{0}'. Any used terms in that last will not be included in the clean up process." +
                        "\nError Details: {1}\nStack Trace: {2}", list.Title, Ex.Message, Ex.StackTrace);
                    Warnings++;
                }
            }//Add the web guid with its lists info
            WebsInfo.Add(web.ID.ToString(), Lists);
            return WebsInfo;
        }
        #endregion
        #region Auxilury Methods
        static string GetToolVersion()
        {
            string Version = "";

            try
            {
                Version CurentVersion = Assembly.GetEntryAssembly().GetName().Version;
                Version = CurentVersion.Major.ToString() + "." + CurentVersion.Minor;
            }
            catch
            {

            }
            return Version;
        }
        static string BuildCamlQuery(List<FieldInfo> Fields)
        {
            StringBuilder FieldsChecks = new StringBuilder();
            string OR = "<Or>{0}</Or>";
            string FieldIsNullCheck = "<IsNotNull><FieldRef Name='{0}' /></IsNotNull>";
            //Prepare the where condition to exclude all items will null values in all MMS fields
            string Where = "<Where>{0}</Where>";

            int Counter = 0;
            foreach (FieldInfo fieldInfoObj in Fields)
            {
                Counter++;
                FieldsChecks.AppendFormat(FieldIsNullCheck, fieldInfoObj.InternalName);
                if (Counter >= 2)
                {
                    FieldsChecks = new StringBuilder(string.Format(OR, FieldsChecks));
                }
            }

            return string.Format(Where, FieldsChecks);
        }
        #endregion
    }
    class FieldInfo
    {
        public FieldInfo(Guid id)
        {
            ID = id;
        }
        public FieldInfo(Guid id, bool valid, string internalName)
        {
            ID = id;
            Valid = valid;
            InternalName = internalName;

        }
        public Guid ID { get; set; }
        public bool Valid { get; set; }
        public string InternalName { get; set; }
    }
}
